/**
 *  @file XmlTokenizer.cpp
 */
#include "XmlTokenizer.h"
#include "../../../convert/Convert.h"
#include "../../../collections/BinBuf.h"
#include <string.h>

using namespace cppflib::io;
using namespace cppflib::collections;

namespace cppflib
{

namespace xml
{

namespace sax
{

XmlTokenizer::XmlTokenizer(void)
{
   this->pStream = NULL;
   this->pTempBuf = NULL;
   this->lineNum = 1;
   this->docTypeCount = 0;
   this->styleSheetCount = 0;
   this->fetchingError = XmlTokenizer::FERR_NONE;
}

XmlTokenizer::XmlTokenizer(Stream *pStream)
{
   const int TEMPBUFSIZE = 256;

   this->pStream = pStream;
   cppmempool::Retain(this->pStream);
   this->pTempBuf = new BinBuf(TEMPBUFSIZE);
   cppmempool::Retain(this->pTempBuf);
   this->lineNum = 1;
   this->docTypeCount = 0;
   this->styleSheetCount = 0;
   this->fetchingError = XmlTokenizer::FERR_NONE;
}

XmlTokenizer::~XmlTokenizer(void)
{
   if (this->pStream)
      cppmempool::Release(this->pStream);
   if (this->pTempBuf)
      cppmempool::Release(this->pTempBuf);
}

void XmlTokenizer::IncrementLineNumber()
{
   this->lineNum++;
}


Stream *XmlTokenizer::GetStream()
{
   return this->pStream;
}

BinBuf *XmlTokenizer::GetTempBuffer()
{
   return this->pTempBuf;
}

int XmlTokenizer::GetLineNumber()
{
   return this->lineNum;
}

int XmlTokenizer::GetDocTypeCount()
{
   return this->docTypeCount;
}
 
int XmlTokenizer::GetStyleSheetCount()
{
   return this->styleSheetCount;
}

XmlTokenizer::fetchingerror_t XmlTokenizer::GetFetchingError()
{
   return this->fetchingError;
}

/////////////////////// tokenizer helper functions //////////////////////////////

PRIVATE void __RewindOneStep(Stream *pStream)
{
   pStream->Seek(-1, Stream::S_CUR);
}

/**
 *  Flush data collected from temp buffer to target token
 */
PRIVATE void __FlushTempBufString(CFString &token, BinBuf &tempBuf)
{
   if (tempBuf.GetValidLength() <= 0)
      return;

   token += reinterpret_cast<const char*>(tempBuf.GetRawBuf());
   tempBuf.ClearBuf();
}

/**
 *  Scan tag name
 */
PRIVATE bool __ScanTagName(XmlTokenizer *tokenizer, CFString &token)
{
   Stream *pStream = tokenizer->GetStream();
   BinBuf *pTempBuf = tokenizer->GetTempBuffer();
   int c;
   u8_t data;
   bool isOK = false;

   pTempBuf->ClearBuf();

   while (true) {
      c = pStream->Read();
      if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '/' || c == '>') {
         if (c == '\n') tokenizer->IncrementLineNumber();
         __RewindOneStep(pStream);
         isOK = true;
         break;
      }
      
      if (c == '.' || c == '-' || c == '_' || c == ':' ||
          (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
          (c >= 0x80 && c <= 0xff)) {  // tag name may have non-English characters
         data = static_cast<u8_t>(c);
         pTempBuf->AppendData(&data, 1, true);  // append to buffer first
      }
      else {
         break;
      }
   }

   if (isOK)
      __FlushTempBufString(token, *pTempBuf);

   return isOK;
}

PRIVATE struct _xmldeclaredtag 
        {
           pcwstr_t tagName;
           wc_t prefixChar;
           XmlTokenizer::tokentype_t type;
        } st_xmlDeclaredTags[] = {
                                   {_S("xml"), _S('?'), XmlTokenizer::XML_DECL_START},
                                   {_S("xml-stylesheet"), _S('?'), XmlTokenizer::XML_STYLESHEET_START},
                                   {_S("DOCTYPE"), _S('!'), XmlTokenizer::XML_DOCTYPE_START},
                                   {NULL, XmlTokenizer::XML_UNKNOWN}
                                 }; 

/**
 *  @param [in] pStream -- stream containing the XML
 *  @param [out] tokenType -- return the token type
 *  @param [out] token -- return the token string
 *
 *  @return true -- a recognizable token is fetched
 */
PRIVATE bool __ScanOpenArrow(XmlTokenizer *tokenizer, XmlTokenizer::tokentype_t &tokenType, CFString &token)
{
   Stream *pStream = tokenizer->GetStream();
   int c = pStream->Read();
   if (c == '?' || c == '!') {
      wc_t prefixC = static_cast<wc_t>(c);

      if (prefixC == '!') {
         c = pStream->Read();
         if (c == '[') { 
            if ((c = pStream->Read()) != 'C') return false;
            if ((c = pStream->Read()) != 'D') return false;
            if ((c = pStream->Read()) != 'A') return false;
            if ((c = pStream->Read()) != 'T') return false;
            if ((c = pStream->Read()) != 'A') return false;
            if ((c = pStream->Read()) != '[') return false;
            tokenType = XmlTokenizer::XML_CDATA_START;
            token = _S("CDATA");
            return true;
         }
         else if (c == '-') {
            if ((c = pStream->Read()) != '-') return false;
            tokenType = XmlTokenizer::XML_COMMENT_START;
            token = _S("--");
            return true;
         }
         else {
            __RewindOneStep(pStream);
         }
      }
      

      if (__ScanTagName(tokenizer, token)) { 
         for (int i = 0; st_xmlDeclaredTags[i].tagName != NULL; ++i) {
            if (prefixC == st_xmlDeclaredTags[i].prefixChar &&
                token == st_xmlDeclaredTags[i].tagName) {
               tokenType = st_xmlDeclaredTags[i].type;
               return true;
             }
         }
      }
   }
   else {
      if (c == '/') {
         c = pStream->Read();
         tokenType = XmlTokenizer::XML_CLOSINGTAGSTART; // may be a closing tag
      }

      if (c == '_' || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
          (c >= 0x80 && c <= 0xff)) {  // tag name may have non-English characters
         __RewindOneStep(pStream);
         if (__ScanTagName(tokenizer, token)) { 
            if (!token.IsEmpty()) {
               if (tokenType == XmlTokenizer::XML_UNKNOWN) 
                  tokenType = XmlTokenizer::XML_TAGSTART;
            
               return true;
            }
         }
      }
   }
    
   return false;      
}

PRIVATE bool __ScanXmlDeclEnd(Stream *pStream, XmlTokenizer::tokentype_t &tokenType, CFString &token)
{
   int c = pStream->Read();
   if (c == '>') {
      tokenType = XmlTokenizer::XML_DECL_END;
      token = "?>";
      return true;
   }
   return false;
}

PRIVATE bool __ScanAttrName(XmlTokenizer *tokenizer, XmlTokenizer::tokentype_t &tokenType, CFString &token)
{
   Stream *pStream = tokenizer->GetStream();
   BinBuf *pTempBuf = tokenizer->GetTempBuffer();
   int c;
   u8_t data;
   bool isOK = false;

   pTempBuf->ClearBuf();

   while (true) {
      c = pStream->Read();
      if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '=' || c == '>' || c == '[') {
         if (c == '\n') tokenizer->IncrementLineNumber();
         tokenType = XmlTokenizer::XML_ATTRNAME;
         __RewindOneStep(pStream);
         isOK = true;
         break;
      }

      if (c == '.' || c == '-' || c == '_' || c == ':' || 
          (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
          (c >= 0x80 && c <= 0xff) ) {  // attribute name may have non-English characters
         data = static_cast<u8_t>(c);
         pTempBuf->AppendData(&data, 1, true);  // append to buffer first
      }
      else {
         break;
      }
   }

   if (isOK)
      __FlushTempBufString(token, *pTempBuf);

   return isOK;
}

PRIVATE struct _xmlpredefinedentity 
        {
           const char * entityName;
           const char * entityVal;
        } st_xmlPredefinedEntityReference[] = {
                                                {"lt", "<"},
                                                {"gt", ">"},
                                                {"amp", "&"},
                                                {"apos", "'"},
                                                {"quot", "\""},
                                                {NULL, NULL}
                                                         }; 

PRIVATE bool __ScanAmpersandEscape(XmlTokenizer *tokenizer, CFString &token)
{
   Stream *pStream = tokenizer->GetStream();
   int c = pStream->Read();
   if (c == '#') {
      wc_t buf[8] = {0};
      int i = 0;
      bool isHex = false;
      
      c = pStream->Read();
      if (c == 'x') {  // may be hex
         buf[i++] = _S('0'); buf[i++] = _S('x');
         isHex = true;
      }
      else if (c >= '0' && c <= '9') {  // may be decimal
         buf[i++] = static_cast<wc_t>(c);
      }
      else {
         if (c == '\n') tokenizer->IncrementLineNumber();
         return false;
      }

      while (i < 7) {
         c = pStream->Read();
         if (c == ';') {
            if (isHex && i <= 2)
               break;

            int n = convert::ToInt32(buf);
            if (n < 0xffff) {
               token += (wc_t)n;
               return true;
            }

            break;
         }
         else if ((c >= '0' && c <= '9') || (isHex && c >= 'A' && c <= 'F') || (isHex && c >= 'a' && c <= 'f'))  {
            buf[i++] = static_cast<wc_t>(c);
         }
         else {
            if (c == '\n') tokenizer->IncrementLineNumber();
            break;
         }
      }
   }
   else {
      char buf[6] = {0};
      int i = 0;

      buf[i++] = static_cast<char>(c);
      while (i < 5) {
         c = pStream->Read();
         if (c < 0) {
            break;
         }
         else if (c == ';') {
            // one of predefined entity ?
            for (int j = 0; st_xmlPredefinedEntityReference[j].entityName != NULL; ++j) {
               if (strcmp(buf, st_xmlPredefinedEntityReference[j].entityName) == 0) {
                  token += st_xmlPredefinedEntityReference[j].entityVal;
                  return true;
               }
            }
            break;
         } 
         else if (c >= 'a' && c <= 'z') {
            buf[i++] = static_cast<char>(c);
         }
         else {
            if (c == '\n') tokenizer->IncrementLineNumber();
            break;
         }
      } // end of while

   }

   return false;
}

/**
 *  '&' and '<' are strictly illegal
 *
 *  @param [in] openQuote -- a single quote or double quote
 */
PRIVATE bool __ScanAttrValue(XmlTokenizer *tokenizer, XmlTokenizer::tokentype_t &tokenType, 
                             CFString &token, int openQuote)
{
   Stream *pStream = tokenizer->GetStream();
   BinBuf *pTempBuf = tokenizer->GetTempBuffer();
   int c, nrWhite = 0;
   u8_t data;
   bool isOK = false;

   pTempBuf->ClearBuf();

   while (true) {
      c = pStream->Read();

      if (c < 0)
         break;

      if (c == ' ' || c == '\t' || c == '\r' || c == '\n') {
         __FlushTempBufString(token, *pTempBuf);
         if (c == '\n') tokenizer->IncrementLineNumber();
         if (nrWhite == 0)
            token += _S(' ');
         ++nrWhite;
         continue;
      }

      nrWhite = 0; // clear

      if (c == openQuote) {
         tokenType = XmlTokenizer::XML_ATTRVALUE;
         isOK = true;
         break;
      }

      if (c == '<') { // illegal character
         break;
      }

      if (c == '&') { // seems to be illegal, check the coming characters
         __FlushTempBufString(token, *pTempBuf);
         if (!__ScanAmpersandEscape(tokenizer, token))
            break;
         else 
            continue;
      }

      data = static_cast<u8_t>(c);
      pTempBuf->AppendData(&data, 1, true);  // append to buffer first

   } // end of while

   if (isOK)
      __FlushTempBufString(token, *pTempBuf);

   return isOK;
}

PRIVATE bool __ScanTagClose_1(Stream *pStream, XmlTokenizer::tokentype_t &tokenType, CFString &token)
{
   int c = pStream->Read();
   if (c == '>') {
      tokenType = XmlTokenizer::XML_TAGCLOSE_1;
      token = "/>";
      return true;
   }
   return false;
}

#define FC_STREAMEND    0
#define FC_CDATAEND   1
#define FC_NOCDATAEND 2

/**
 *  @return 0 -- end of stream
 *  @return 1 -- CDATA ending pattern ']]>' is found
 *  @return 2 -- not found
 */
PRIVATE int __IsCDATAEndingPatternFound(XmlTokenizer *tokenizer)
{
   Stream *pStream = tokenizer->GetStream();
   BinBuf *pTempBuf = tokenizer->GetTempBuffer();
   int c, nrByteRead = 0;
   u8_t data;
   
   
   // read 2 more characters
   do {
      ++nrByteRead;
      c = pStream->Read();
      if (c < 0) return FC_STREAMEND;
      data = static_cast<u8_t>(c);
      pTempBuf->AppendData(&data, 1, true);  // append to buffer first
      if (c != ']') break;

      ++nrByteRead;
      c = pStream->Read();
      if (c < 0) return FC_STREAMEND;
      data = static_cast<u8_t>(c);
      pTempBuf->AppendData(&data, 1, true);  // append to buffer first
      if (c != '>') break;
   
      return FC_CDATAEND; // ']]>' is found

   } while(false);

   // rewind
   pTempBuf->SetValidLength(pTempBuf->GetValidLength() - nrByteRead);
   for (int i = 0; i < nrByteRead; ++i)
      __RewindOneStep(pStream);

   return FC_NOCDATAEND;
}


/**
 *  Fetch a token
 *
 *  @param [out] token -- output token
 *  @param [in] state -- parsing state
 *
 *  @return type of token parsed
 */
XmlTokenizer::tokentype_t XmlTokenizer::GetToken(CFString &token)
{
   XmlTokenizer::tokentype_t tokenType = XmlTokenizer::XML_UNKNOWN;
   bool isScanOK = false;
   int nrWhite = 0;

   token = ""; // clear first

   while(true) {
      int c = pStream->Read();

      if (c < 0) {
         isScanOK = true;
         tokenType = XmlTokenizer::XML_STREAMEND;
         token = "";
         break;
      }
      else if (c == ' ' || c == '\t' || c == '\r' || c == '\n') {
         if (c == '\n') this->IncrementLineNumber();
         ++nrWhite;
      }
      else if (c == '<') {
         isScanOK = __ScanOpenArrow(this, tokenType, token);

         if (isScanOK) {
            if (tokenType == XmlTokenizer::XML_DECL_START && nrWhite > 0) {
               // should have no space before XML_DECL_START
               tokenType = XmlTokenizer::XML_ERROR_XML_DECL_START;
               token = "";
            }
            else if (tokenType == XmlTokenizer::XML_DOCTYPE_START) {
               this->docTypeCount++;
            }
            else if (tokenType == XmlTokenizer::XML_STYLESHEET_START) {
               this->styleSheetCount++;
            }
         }
         break;
      }
      else if (c == '?') {
         isScanOK = __ScanXmlDeclEnd(pStream, tokenType, token);
         break;
      }
      else if (c == '=') {
         isScanOK = true;
         tokenType = XmlTokenizer::XML_EQU;
         token = "=";
         break;
      }
      else if (c == '/') {
         isScanOK = __ScanTagClose_1(pStream, tokenType, token);
         break;
      }
      else if (c == '>') {
         isScanOK = true;
         tokenType = XmlTokenizer::XML_TAGCLOSE_2;
         token = ">";
         break;
      }
      else if (c == '[') {
         isScanOK = true;
         tokenType = XmlTokenizer::XML_KEYWORD_OPENSQUAREBRACKET;
         token = "[";
         break;
      }
      else if (c == '\'' || c == '"') {
         isScanOK = __ScanAttrValue(this, tokenType, token, c);
         break;
      }
      else if (c == '_' || c == ':' || 
               (c >= 'A' && c <= 'Z') || 
               (c >= 'a' && c <= 'z') || 
               (c >= 0x80 && c <= 0xff)) { // attribute name may have non-English characters
         __RewindOneStep(pStream);
         isScanOK = __ScanAttrName(this, tokenType, token);
         break;
      }
      else {
         break; // unknown
      }

   } // end of while

   if (!isScanOK) {
      // clear
      tokenType = XmlTokenizer::XML_UNKNOWN;
      token = "";
   }

       
   return tokenType;
}

/**
 *  Collect string content in between XML open and close tag
 *  i.e.  <hello>string content</hello>
 *
 *  '&' and '<' are strictly illegal
 *  
 *  This one is similar to __ScanAttrValue but with a little difference
 *  
 *  @return true -- ok
 *  @return false -- something wrong
 */
bool XmlTokenizer::CollectElementContent(CFString &content)
{
   int c, ret;
   u8_t data;
   bool isOK = false;

   content = "";
   pTempBuf->ClearBuf();

   while (true) {
      c = pStream->Read();
      if (c < 0)
         break;

      if (c == '<') { 
         isOK = true;
         __RewindOneStep(pStream);
         break;
      }

      if (c == '&') { // seems to be illegal, check the coming characters
         __FlushTempBufString(content, *pTempBuf);
         if (!__ScanAmpersandEscape(this, content))
            break;
         else 
            continue;
      }

      if (c == '\n') 
         this->IncrementLineNumber();

      data = static_cast<u8_t>(c);
      pTempBuf->AppendData(&data, 1, true);  // append to buffer first

      if (c == ']') {
         ret = __IsCDATAEndingPatternFound(this);
         if (ret == FC_STREAMEND) {
            break;
         }
         else if (ret == FC_CDATAEND) {
            // CDATA ending pattern found, which is disallowed
            this->fetchingError = XmlTokenizer::FERR_CDATA_END_DISALLOW;
            break;
         }
      }

   } // end of while

   if (isOK)
      __FlushTempBufString(content, *pTempBuf);

   return isOK;
}

/**
 *  Collect string content inside CDATA
 */
bool XmlTokenizer::CollectCDATAContent(CFString &content)
{
   int c, ret;
   u8_t data;
   bool isOK = false;

   content = "";
   pTempBuf->ClearBuf();

   while (true) {
      c = pStream->Read();
      if (c < 0) break;
      if (c == '\n') this->IncrementLineNumber();
      data = static_cast<u8_t>(c);
      pTempBuf->AppendData(&data, 1, true);  // append to buffer first
      if (c != ']') continue;

      ret = __IsCDATAEndingPatternFound(this);
      if (ret == FC_STREAMEND) break;
      else if (ret == FC_NOCDATAEND) continue;
      
      // ']]>' is matched
      pTempBuf->SetValidLength(pTempBuf->GetValidLength() - 3); // cut the trailing ']]>'
      isOK = true;
      break;

   } // end of while

   if (isOK)
      __FlushTempBufString(content, *pTempBuf);

   return isOK;
}


} // end of namespace sax

} // end of namespace xml

} // end of namespace cppflib
